# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import re
from scrapy.exceptions import DropItem
from scrapy.pipelines.files import FilesPipeline
from scrapy.pipelines.images import ImagesPipeline
from scrapy import Request

class ScrapyexercisePipeline:
    file_name = None
    def process_item(self, item, spider):
        if item["form"] == "连载":
            item["form"] = "LZ"
        else:
            raise DropItem("抛弃该item")
        return item
    def open_spider(self, spider):
        #当Spider开启时（爬取数据之前），该方法被调用。
        pass
    def close_spider(self, spider):
        #当Spider关闭时（所有数据被爬取完毕），该方法被调用。
        pass
    @classmethod
    def from_crawler(cls, crawler):
        #该方法被调用时，会创建一个新的item pipline对象，参数crawler为使用当前管道的项目
        cls.file_name = crawler.settings.get("FILE_NAME", 'hot.txt')
        return cls()

class FilterPipeline(object):
    def process_item(self, item, spider):
        item['area'] = re.findall(r"\d+\.?\d*", item['area'])[0]
        if item['direction'] == "暂无数据":
            raise DropItem("房屋朝向无数据，抛弃此项目")
        return item

class CSVPipeline(object):
    index = 0
    file = None
    def open_spider(self, spider):
        self.file = open("home.csv", "a", encoding="utf-8")

    def process_item(self, item, spider):
        if self.index == 0:
            column_name = "name,type,area,direction,fitment,elevator,total_price,unit_price,property\n"
            self.file.write(column_name)
            self.index = 1
        home_str = item['name']+","+\
                   item['type']+","+ \
                   item['area'] + "," + \
                   item['direction'] + "," + \
                   item['fitment'] + "," + \
                   item['elevator'] + "," + \
                   item['total_price'] + "," + \
                   item['unit_price'] + "," + \
                   item['property'] + "\n"
        self.file.write(home_str)
        return item

    def close_spider(self, spider):
        self.file.close()

class SaveFilesPipeline(FilesPipeline):

    # 设置文件名称
    def file_path(self, request, response=None, info=None, *, item=None):
        # return request.url.split("/")[-1]
        #设置文件夹和文件组合的字符串
        return "folder/" + request.url.split("/")[-1]

class SaveImagePipeline(ImagesPipeline):
    # 构造图像下载的请求
    def get_media_requests(self, item, info):
        # 将图片主题传递进去用于设置存储图片的存储路径
        return [Request(x, meta={"title":item["title"]}) for x in item.get(self.images_urls_field, [])]

    # 设置照片存储路径及名称
    def file_path(self, request, response=None, info=None, *, item=None):
        title = request.meta['title']
        image_name = request.url.split("/")[-1]
        return "%s/%s" % (title, image_name)

    # 设置缩略图存储路径及名称
    def thumb_path(self, request, thumb_id, response=None, info=None):
        title = request.meta['title']
        image_name = request.url.split("/")[-1]
        return "%s/%s/%s" % (title, thumb_id, image_name)